{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Script for extracting clean temperature data from BigQuery\n",
    "This runs on datalab.\n",
    "The end result are the 1666 weather station files each containing min and max daily temperature data spanning 50 years.\n",
    " - from 1960-01-02\n",
    " - to 2009-12-31 included.\n",
    " \n",
    "All files have the exact same number of entries and the same date spans. Some interpolation\n",
    "was necessary to fill measurement gaps in the original data. Filled-in data points are marked as such in the \"interpolated\" column. Weather stations were selected based on the following criteria:\n",
    " - no more than 300 gaps\n",
    " - no more than 30 consecutive gaps\n",
    " \n",
    "The script runs for a long time. It also has a tendency to fail after a whie and must be restarted. It will skip stations that have already been downloaded but it will retry stations that have been discarded as incomplete. You can fill in the list of discarded stations in 'stations_skip' to avoid re-loading and discarding them again.\n",
    "\n",
    "***Results are now available on a public GCS bucket: gs://good-temperatures-public***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from __future__ import print_function\n",
    "import google.datalab.bigquery as bq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATADIR = \"good_temperature_data\"\n",
    "if not os.path.exists(DATADIR):\n",
    "  os.mkdir(DATADIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%bq query --name daily_tminmax\n",
    "# Format weather station data between 1960 and 2010\n",
    "# as daily min and max temperatures\n",
    "SELECT stationid, date, daystamp,\n",
    "      MIN(tmin) AS tmini,\n",
    "      MAX(tmax) AS tmaxi\n",
    "    FROM(\n",
    "      SELECT\n",
    "        wx.id AS stationid,\n",
    "        wx.date AS date,\n",
    "        wx.element AS element,\n",
    "        wx.value AS value,\n",
    "        UNIX_SECONDS(CAST(wx.date AS TIMESTAMP))/(24*3600) AS daystamp,\n",
    "        IF (wx.element = 'TMIN', wx.value/10, NULL) AS tmin,\n",
    "        IF (wx.element = 'TMAX', wx.value/10, NULL) AS tmax\n",
    "      FROM\n",
    "        (SELECT * FROM `bigquery-public-data.ghcn_d.ghcnd_1*` UNION ALL\n",
    "        SELECT * FROM `bigquery-public-data.ghcn_d.ghcnd_2*`) AS wx\n",
    "      WHERE\n",
    "        #id = 'USC00457478' AND\n",
    "        (wx.element ='TMIN'OR wx.element ='TMAX')\n",
    "    )\n",
    "    WHERE\n",
    "      daystamp < 14610      # year 2010\n",
    "      AND daystamp > -3653  # year 1960\n",
    "    GROUP BY\n",
    "      stationid, date, daystamp\n",
    "    HAVING tmini IS NOT NULL AND tmaxi IS NOT NULL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%bq query --name good_stations --subqueries daily_tminmax\n",
    "# select stations that have an almost uninterrupted series of data records\n",
    "# allow 100 holes max\n",
    "SELECT stationid, MIN(daystamp) AS minday, MAX(daystamp) AS maxday, COUNT(tmini) AS count_mini, count(tmaxi) AS count_maxi,\n",
    "       MAX(daystamp)-MIN(daystamp)-COUNT(tmini) AS nb_holes, MAX(daystamp)-MIN(daystamp) AS nb_days\n",
    "FROM daily_tminmax\n",
    "GROUP BY stationid\n",
    "HAVING nb_holes <300 AND nb_days >= 50*365-300"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%bq query --name get_one_station --subqueries daily_tminmax good_stations\n",
    "# Select weather stations with an at least 50-year measurement span (1960-2010)\n",
    "# and a maximum of 100 measurement gaps. This yields 1221 weather stations and\n",
    "# approx. 22M days of min-max temperature data.\n",
    "\n",
    "# Get daily min-max data from all these stations\n",
    "SELECT *\n",
    "FROM daily_tminmax\n",
    "WHERE\n",
    "  daily_tminmax.stationid = @thisstation AND\n",
    "  daily_tminmax.stationid IN (SELECT stationid FROM good_stations)\n",
    "ORDER BY daystamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%bq query --name good_stationids --subqueries good_stations\n",
    "SELECT stationid\n",
    "FROM good_stations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'ASN00014015'\n",
    "# this one does not work. Gets a \"undeclared query parameters\" argument\n",
    "#df = get_one_station.execute(output_options=bq.QueryOutput.dataframe(), query_params={'thisstation':'ASN00014015'}).result()\n",
    "#df.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_gaps(dfmm, maxgaps):\n",
    "  i = 1\n",
    "  prevday = dfmm.iloc[0]['daystamp']\n",
    "  gaps = [0 for _ in range(maxgaps+1)]\n",
    "  for _, row in dfmm.iterrows():\n",
    "    day = row['daystamp']\n",
    "    if i > 0:\n",
    "      for k in range(maxgaps):\n",
    "        if day - prevday == k+2:\n",
    "          gaps[k] += 1\n",
    "      if day - prevday > maxgaps+1:\n",
    "        gaps[maxgaps] += 1\n",
    "    prevday = day\n",
    "    i+=1\n",
    "    \n",
    "  return gaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stations_df = good_stationids.execute(output_options=bq.QueryOutput.dataframe()).result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scan directory for files that have already been processed\n",
    "stations_done = []\n",
    "for filename in os.listdir(DATADIR):\n",
    "  print(filename[:-4])\n",
    "  stations_done.append(filename[:-4])\n",
    "# skip stations that have already been done\n",
    "stations_todo = stations_df.loc[~stations_df['stationid'].isin(stations_done)]\n",
    "# skip additional stations\n",
    "stations_skip = []\n",
    "stations_todo = stations_todo.loc[~stations_todo['stationid'].isin(stations_skip)]\n",
    "stations_todo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "LARGEST_GAP = 31\n",
    "for i, stationid in stations_todo.iterrows():\n",
    "  sid = stationid['stationid']\n",
    "  print(sid)\n",
    "  q = get_one_station.sql.replace('@thisstation', \"\\'\"+sid+\"\\'\")\n",
    "  df = bq.Query(q).execute_async(output_options=bq.QueryOutput.dataframe()).result()\n",
    "  gaps = find_gaps(df, maxgaps=LARGEST_GAP+1)\n",
    "  print(gaps)\n",
    "  if gaps[LARGEST_GAP+1] == 0:\n",
    "    # interpolate missing values\n",
    "    # but only on stations where the gaps are no larger than LARGEST_GAP days\n",
    "    ddf = df.set_index(pd.DatetimeIndex(df['date']))\n",
    "    ddf = ddf.resample(\"1D\").interpolate()\n",
    "    ddf = ddf.assign(interpolated=pd.isnull(ddf['stationid']).astype(int)) # mark interpolated values as such\n",
    "    print(find_gaps(ddf, maxgaps=LARGEST_GAP+1))\n",
    "    ddf.to_csv(DATADIR+\"/\"+sid+\".csv\", columns=['tmini', 'tmaxi', 'interpolated']) # and also the date\n",
    "  else:\n",
    "    print(\"Gaps too large. Dropped.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Additional head and tail cleaning\n",
    "Lists files that are missing only a few lines at the begining or the end.\n",
    "You can add the missing lines manually by duplicating the first/last lines\n",
    "and marking them as \"interpolated\". This code only lists the files, all\n",
    "corrections are manual."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................."
     ]
    }
   ],
   "source": [
    "bad_files=[]\n",
    "good_files = []\n",
    "recoverable_files=[]\n",
    "for filename in os.listdir(DATADIR):\n",
    "  with open(DATADIR + \"/\" + filename, mode=\"r\") as f:\n",
    "    line = f.readline()\n",
    "    linenb = 1\n",
    "    while line:\n",
    "      line = f.readline()\n",
    "      linenb += 1  \n",
    "      if linenb == 2:\n",
    "        firstline = line\n",
    "    # Good ?\n",
    "    if linenb == 18264 and firstline[:10] == \"1960-01-02\":\n",
    "      good_files.append(filename)\n",
    "      print('.', end='')\n",
    "    # Bad ?\n",
    "    else:\n",
    "      print()\n",
    "      print(linenb)\n",
    "      print(firstline, end='')\n",
    "      bad_files.append(filename)\n",
    "      # Recoverable ?\n",
    "      if (linenb == 18264 or\n",
    "          linenb == 18263 or\n",
    "          linenb == 18262 or\n",
    "          linenb == 18261):\n",
    "        if (firstline[:10] == \"1960-01-02\" or\n",
    "            firstline[:10] == \"1960-01-03\" or\n",
    "            firstline[:10] == \"1960-01-04\" or\n",
    "            firstline[:10] == \"1960-01-05\"):\n",
    "          print(\"recoverable\")\n",
    "          recoverable_files.append(bad_files.pop()) # transfer it to recoverable list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bad files: 0\n",
      "[]\n",
      "Recoverable files: 0\n",
      "[]\n",
      "Good files: 1666\n",
      "['USW00003822.csv', 'USW00023258.csv', 'RSM00028537.csv', 'USC00215598.csv', 'RSM00027823.csv', 'RSM00030393.csv', 'USC00163313.csv', 'RSM00023884.csv', 'RSM00024641.csv', 'NOE00109514.csv', 'USC00416792.csv', 'CHM00050527.csv', 'CHM00059265.csv', 'USW00023072.csv', 'CHM00054027.csv', 'USC00425402.csv', 'RSM00025563.csv', 'USW00023066.csv', 'USC00053005.csv', 'USC00295617.csv', 'USC00124272.csv', 'USW00024090.csv', 'USC00407884.csv', 'USC00203661.csv', 'USC00315771.csv', 'NO000050540.csv', 'USC00113384.csv', 'USC00352374.csv', 'CHM00052495.csv', 'CA004028060.csv', 'GMW00034068.csv', 'USC00352406.csv', 'USC00233601.csv', 'USW00093821.csv', 'CHM00056778.csv', 'USW00024133.csv', 'USW00024127.csv', 'CHM00053614.csv', 'CHM00051765.csv', 'CHM00058027.csv', 'USC00332251.csv', 'AU000011801.csv', 'USW00014755.csv', 'CA004020560.csv', 'USC00134101.csv', 'CA007060400.csv', 'USW00013748.csv', 'USC00298535.csv', 'USC00355221.csv', 'IT000016320.csv', 'USC00292030.csv', 'ROE00100829.csv', 'RSM00030230.csv', 'CHM00051573.csv', 'CHM00054423.csv', 'USC00051564.csv', 'FIE00143066.csv', 'FMW00040308.csv', 'USC00395561.csv', 'SPE00120458.csv', 'USC00391049.csv', 'SZ000003700.csv', 'USC00396054.csv', 'USC00417704.csv', 'USC00419715.csv', 'USC00410902.csv', 'USC00364214.csv', 'CA004016560.csv', 'USW00026533.csv', 'USC00101408.csv', 'RSM00027995.csv', 'ASN00023034.csv', 'USC00415477.csv', 'USW00014768.csv', 'USC00215638.csv', 'USW00014740.csv', 'CA001018620.csv', 'USW00013985.csv', 'USC00338769.csv', 'USC00204090.csv', 'GME00120934.csv', 'USC00181750.csv', 'USC00148578.csv', 'USW00024132.csv', 'USW00093820.csv', 'USW00094008.csv', 'GM000004155.csv', 'USC00415271.csv', 'USC00399004.csv', 'USC00340017.csv', 'USC00231822.csv', 'USW00094746.csv', 'AU000005901.csv', 'GME00111445.csv', 'CHM00054026.csv', 'USC00338609.csv', 'RSM00030635.csv', 'SWE00140554.csv', 'RSM00023891.csv', 'USC00041428.csv', 'SWE00137910.csv', 'RSM00034321.csv', 'CHM00052866.csv', 'USC00143954.csv', 'LOE00116364.csv', 'USC00473453.csv', 'USW00023271.csv', 'USC00102707.csv', 'RSM00030437.csv', 'RSE00152340.csv', 'CHM00054218.csv', 'FIE00144132.csv', 'USC00146637.csv', 'RSM00034123.csv', 'CHM00057707.csv', 'GME00102177.csv', 'USC00417841.csv', 'CHM00057713.csv', 'NOE00109701.csv', 'USW00026451.csv', 'USW00023273.csv', 'USC00147271.csv', 'USC00136305.csv', 'CHM00050915.csv', 'FIE00141935.csv', 'USC00129557.csv', 'USC00114198.csv', 'USC00428119.csv', 'FR000007255.csv', 'USC00235307.csv', 'USW00014839.csv', 'USW00023065.csv', 'RSM00028722.csv', 'CA006142400.csv', 'USC00213303.csv', 'USC00422385.csv', 'CHM00056182.csv', 'USC00025635.csv', 'USC00097087.csv', 'ASN00086071.csv', 'USC00324418.csv', 'USC00033862.csv', 'USC00355593.csv', 'USW00093822.csv', 'USC00227111.csv', 'USW00024130.csv', 'USC00293488.csv', 'CHM00056021.csv', 'USW00014742.csv', 'USC00405882.csv', 'USC00248501.csv', 'USC00132573.csv', 'USW00013987.csv', 'USC00348501.csv', 'SZ000008440.csv', 'EN000026242.csv', 'CA003025480.csv', 'ASN00015590.csv', 'USC00429165.csv', 'USC00116610.csv', 'RSM00030555.csv', 'CHM00054346.csv', 'USW00021504.csv', 'SWE00137494.csv', 'RSM00028440.csv', 'CA008501100.csv', 'USC00147305.csv', 'ASN00087031.csv', 'USC00130600.csv', 'USC00143822.csv', 'SPE00120467.csv', 'USC00391076.csv', 'RSM00029789.csv', 'GQW00041415.csv', 'ASN00002012.csv', 'RSM00031845.csv', 'CHM00059663.csv', 'RSM00030554.csv', 'FR000007650.csv', 'USC00402108.csv', 'RSM00030965.csv', 'USC00385200.csv', 'USW00094143.csv', 'USC00148235.csv', 'RSM00022408.csv', 'USC00477226.csv', 'FRE00104907.csv', 'UZM00038413.csv', 'USC00145063.csv', 'UK000003162.csv', 'USC00339312.csv', 'USC00392797.csv', 'USW00024131.csv', 'NOE00109476.csv', 'USC00342944.csv', 'RSM00030811.csv', 'USC00351643.csv', 'RSM00022583.csv', 'USW00094745.csv', 'USC00134705.csv', 'USC00088824.csv', 'FIE00144482.csv', 'USC00079605.csv', 'USC00145856.csv', 'CHM00059501.csv', 'USW00023070.csv', 'GME00102348.csv', 'NOE00105476.csv', 'RSM00024125.csv', 'SP000008181.csv', 'GME00102412.csv', 'RSM00034861.csv', 'CHM00052681.csv', 'RSM00030385.csv', 'USC00131319.csv', 'USW00024284.csv', 'SPE00120512.csv', 'CHM00050727.csv', 'USC00296687.csv', 'USW00023272.csv', 'RIE00111909.csv', 'RSM00027612.csv', 'PSW00040309.csv', 'USC00314860.csv', 'RSM00034646.csv', 'USW00003820.csv', 'USC00410691.csv', 'RIE00100814.csv', 'USC00478267.csv', 'ASN00039123.csv', 'USC00143984.csv', 'USW00024243.csv', 'USC00114823.csv', 'RSM00031909.csv', 'USC00324958.csv', 'USW00094225.csv', 'RSM00031510.csv', 'NOE00105472.csv', 'SPE00101043.csv', 'CHM00058633.csv', 'LO000011934.csv', 'RSM00022550.csv', 'GME00102364.csv', 'CA002401200.csv', 'CHM00053772.csv', 'RSM00030815.csv', 'SWE00137724.csv', 'USC00316256.csv', 'USC00123527.csv', 'USC00253050.csv', 'RSM00028900.csv', 'USC00132364.csv', 'USW00013969.csv', 'USC00245961.csv', 'USC00206680.csv', 'CA004018760.csv', 'ASN00066037.csv', 'USW00024121.csv', 'CHM00051777.csv', 'USW00024135.csv', 'USC00245020.csv', 'USW00013996.csv', 'RSM00023921.csv', 'USC00422592.csv', 'FI000007501.csv', 'RSM00030949.csv', 'FIE00145377.csv', 'CA003081680.csv', 'USC00011620.csv', 'USC00192501.csv', 'USC00318694.csv', 'CHM00057662.csv', 'CA008201410.csv', 'NOE00110447.csv', 'USC00456039.csv', 'USC00412121.csv', 'CHM00059855.csv', 'USC00215400.csv', 'USC00445050.csv', 'RSM00028493.csv', 'USC00458773.csv', 'AU000016402.csv', 'USC00357169.csv', 'NOE00109671.csv', 'USC00226009.csv', 'USC00119241.csv', 'USC00426534.csv', 'USC00238223.csv', 'USC00456789.csv', 'CHM00054342.csv', 'SP000008027.csv', 'CHM00056964.csv', 'CHM00057461.csv', 'RSM00034247.csv', 'USW00023129.csv', 'RSM00029570.csv', 'USW00013968.csv', 'CA006073975.csv', 'SWE00137692.csv', 'SWE00139720.csv', 'USC00243530.csv', 'USC00136103.csv', 'USC00478919.csv', 'ROE00108889.csv', 'RSM00030627.csv', 'FIE00144322.csv', 'NOE00105467.csv', 'USW00094967.csv', 'USC00253910.csv', 'GME00111457.csv', 'SPE00120098.csv', 'USW00023061.csv', 'RSM00035011.csv', 'USW00094224.csv', 'SPE00120503.csv', 'CHM00058424.csv', 'USC00105708.csv', 'RSM00031707.csv', 'USC00334551.csv', 'CHM00054236.csv', 'ASN00025509.csv', 'IT000016090.csv', 'CA008401700.csv', 'USW00012916.csv', 'USC00218907.csv', 'CHM00054208.csv', 'USC00128698.csv', 'CHM00057067.csv', 'USC00414382.csv', 'FIE00144877.csv', 'RSM00034866.csv', 'CA001183000.csv', 'RSM00031088.csv', 'RSM00029348.csv', 'UK000003005.csv', 'RSM00024136.csv', 'CHM00057265.csv', 'FIE00142751.csv', 'NLE00102114.csv', 'USC00249023.csv', 'USW00093987.csv', 'RSM00023711.csv', 'USC00015121.csv', 'USC00036920.csv', 'CHM00052533.csv', 'NOE00109459.csv', 'USW00093824.csv', 'USW00013942.csv', 'RSM00031329.csv', 'RSM00030745.csv', 'GM000003038.csv', 'USW00014778.csv', 'USC00252205.csv', 'USC00312635.csv', 'USC00052281.csv', 'USW00014750.csv', 'ASN00012038.csv', 'USW00013995.csv', 'ASN00076031.csv', 'GME00121186.csv', 'USW00026523.csv', 'USC00152040.csv', 'FIE00142341.csv', 'CA002202800.csv', 'NOE00109840.csv', 'RSM00022802.csv', 'USC00138266.csv', 'ASN00039083.csv', 'USC00129430.csv', 'RSM00027947.csv', 'EI000003953.csv', 'USC00131257.csv', 'RSM00034720.csv', 'FR000007130.csv', 'CHM00058221.csv', 'USC00132789.csv', 'USC00027390.csv', 'USC00107040.csv', 'RSM00023274.csv', 'NOM00001492.csv', 'USW00014745.csv', 'USW00013994.csv', 'USC00317994.csv', 'NOE00110653.csv', 'CHM00056768.csv', 'USW00013957.csv', 'USW00024137.csv', 'GME00102212.csv', 'USC00292785.csv', 'RMW00040710.csv', 'CHM00054157.csv', 'USW00093819.csv', 'CA008403506.csv', 'USC00465963.csv', 'ASN00045015.csv', 'USW00093986.csv', 'SP000006155.csv', 'GME00102372.csv', 'RSM00028719.csv', 'USC00138688.csv', 'USW00024057.csv', 'USC00327704.csv', 'CHM00057516.csv', 'USC00166938.csv', 'CHM00056146.csv', 'USW00023062.csv', 'USC00057515.csv', 'SWE00115961.csv', 'SZ000002220.csv', 'USC00313106.csv', 'CA002100880.csv', 'RSM00027199.csv', 'GME00121018.csv', 'RSM00029822.csv', 'ASN00009021.csv', 'USC00335857.csv', 'CA008205700.csv', 'RSM00023472.csv', 'USC00123082.csv', 'USC00406977.csv', 'USW00012917.csv', 'USC00429717.csv', 'USW00024232.csv', 'USW00026425.csv', 'CHM00053529.csv', 'USC00296619.csv', 'USC00446475.csv', 'RSM00031788.csv', 'SP000009981.csv', 'CHM00056492.csv', 'USC00102942.csv', 'CHM00050949.csv', 'RSM00027459.csv', 'USC00104670.csv', 'RSM00021982.csv', 'USC00474174.csv', 'USW00024018.csv', 'USC00340908.csv', 'USW00013878.csv', 'USW00013893.csv', 'HRE00105217.csv', 'USC00402009.csv', 'USC00242104.csv', 'USC00107386.csv', 'USW00022521.csv', 'USW00024150.csv', 'USW00093842.csv', 'CHM00057411.csv', 'IR000408560.csv', 'RSM00028144.csv', 'SPE00120188.csv', 'LH000026509.csv', 'RSM00034579.csv', 'RSM00024959.csv', 'USC00034638.csv', 'CHM00056096.csv', 'RSM00025138.csv', 'USC00482580.csv', 'RSM00030521.csv', 'CHM00050632.csv', 'RSM00034009.csv', 'USC00475120.csv', 'RSM00028434.csv', 'CA001163780.csv', 'USC00137669.csv', 'CHM00054497.csv', 'USC00345509.csv', 'USC00420849.csv', 'USC00442941.csv', 'SW000002590.csv', 'USW00012839.csv', 'SWE00139396.csv', 'CA001100030.csv', 'CHM00053845.csv', 'GME00120994.csv', 'USC00218419.csv', 'USW00014737.csv', 'USW00025501.csv', 'FIE00145677.csv', 'USW00023170.csv', 'USC00188065.csv', 'USC00374266.csv', 'RSM00023986.csv', 'GM000010962.csv', 'USC00218579.csv', 'USC00131635.csv', 'RSM00030695.csv', 'USC00251145.csv', 'RSM00030859.csv', 'USW00094725.csv', 'USC00411048.csv', 'USC00157215.csv', 'RSM00034356.csv', 'USC00054734.csv', 'RSM00029313.csv', 'USW00024025.csv', 'USC00115768.csv', 'CHM00057799.csv', 'USC00356366.csv', 'USC00422864.csv', 'USC00203429.csv', 'GME00121042.csv', 'USC00052184.csv', 'CHM00056444.csv', 'RSM00025703.csv', 'USC00331458.csv', 'RSM00031004.csv', 'USW00024233.csv', 'SZ000001940.csv', 'USW00024227.csv', 'USW00003856.csv', 'CHM00058457.csv', 'CHM00053502.csv', 'USC00420730.csv', 'USW00024225.csv', 'ROE00100901.csv', 'USC00048135.csv', 'CHM00050745.csv', 'RSM00027857.csv', 'SIE00105938.csv', 'CHM00054292.csv', 'CHM00057957.csv', 'USC00319467.csv', 'USW00003103.csv', 'USW00024033.csv', 'RSM00034432.csv', 'GME00102316.csv', 'UPM00033999.csv', 'USW00025339.csv', 'USC00475255.csv', 'RSM00037107.csv', 'SW000002196.csv', 'CHM00059211.csv', 'ASN00072150.csv', 'CHM00052436.csv', 'USC00165266.csv', 'USC00354003.csv', 'USW00022536.csv', 'GME00102276.csv', 'USW00024153.csv', 'USW00025503.csv', 'USC00475335.csv', 'RSM00031416.csv', 'CHM00059417.csv', 'USW00014735.csv', 'USC00241552.csv', 'USC00253630.csv', 'USC00133632.csv', 'NLE00102011.csv', 'USC00313228.csv', 'GME00120982.csv', 'AU000006306.csv', 'USC00476827.csv', 'USC00469522.csv', 'CHM00058251.csv', 'UK000003302.csv', 'CHM00052754.csv', 'HRE00105189.csv', 'USC00219249.csv', 'USC00043855.csv', 'GM000001474.csv', 'USC00455224.csv', 'USC00131233.csv', 'USC00414278.csv', 'SW000002550.csv', 'USC00121747.csv', 'USC00161411.csv', 'FIE00143952.csv', 'USW00004725.csv', 'RSE00151775.csv', 'RSM00023205.csv', 'USC00214793.csv', 'CHM00054324.csv', 'USC00100010.csv', 'CHM00053463.csv', 'RSM00022641.csv', 'CHM00056080.csv', 'USC00127646.csv', 'USC00350304.csv', 'CA001108447.csv', 'USC00248211.csv', 'USW00014734.csv', 'USW00093129.csv', 'USC00051713.csv', 'RSM00031371.csv', 'USW00014913.csv', 'USW00094849.csv', 'CHM00051076.csv', 'USC00208184.csv', 'USW00014898.csv', 'USW00013891.csv', 'USC00448894.csv', 'RSM00022292.csv', 'CHM00056651.csv', 'UZM00038262.csv', 'USW00023007.csv', 'USC00451484.csv', 'ASN00033119.csv', 'USW00024032.csv', 'ASN00014015.csv', 'USC00300889.csv', 'SP000007038.csv', 'CA003031093.csv', 'NO000080700.csv', 'USC00318744.csv', 'CHM00050963.csv', 'CA008104400.csv', 'CA004048520.csv', 'ROE00100900.csv', 'RSM00027665.csv', 'IC000004048.csv', 'USC00053500.csv', 'FR000007747.csv', 'NOE00109777.csv', 'RSM00030309.csv', 'RSM00027675.csv', 'RSM00028552.csv', 'RSM00037472.csv', 'FI000002963.csv', 'RSM00029664.csv', 'USW00003847.csv', 'CHM00059758.csv', 'USC00393029.csv', 'USC00125337.csv', 'USC00478589.csv', 'ASN00007045.csv', 'USC00143930.csv', 'USC00117636.csv', 'RSM00029328.csv', 'USC00283951.csv', 'USW00024036.csv', 'USW00024022.csv', 'USC00242857.csv', 'CA004012400.csv', 'USC00300443.csv', 'RSM00024817.csv', 'USC00344573.csv', 'USC00216547.csv', 'USC00212721.csv', 'USW00013881.csv', 'USW00013895.csv', 'FIE00142080.csv', 'USC00326255.csv', 'USC00136940.csv', 'USC00339361.csv', 'USW00024156.csv', 'CA002403500.csv', 'ASN00006011.csv', 'USC00232591.csv', 'USC00030458.csv', 'NLM00006380.csv', 'USW00023188.csv', 'USW00013739.csv', 'USC00241557.csv', 'USC00032930.csv', 'USW00003927.csv', 'CA001092970.csv', 'RSM00029923.csv', 'USC00328913.csv', 'SWE00138058.csv', 'USC00152791.csv', 'USC00187806.csv', 'CHM00052787.csv', 'ASN00016001.csv', 'ASN00044021.csv', 'USC00051528.csv', 'USC00040343.csv', 'USC00230608.csv', 'CHM00058527.csv', 'LH000026730.csv', 'RSM00034740.csv', 'USC00255090.csv', 'USC00248957.csv', 'USC00127482.csv', 'RSM00027066.csv', 'SPE00120602.csv', 'USW00024157.csv', 'USC00146242.csv', 'USC00414081.csv', 'USC00253185.csv', 'USW00024143.csv', 'CHM00056046.csv', 'USW00025507.csv', 'USC00031948.csv', 'USW00013894.csv', 'ASN00009519.csv', 'CA006037775.csv', 'HRE00105210.csv', 'CHM00054916.csv', 'USW00093058.csv', 'RSM00021946.csv', 'USW00013880.csv', 'USC00319675.csv', 'RSM00024802.csv', 'CHM00054094.csv', 'USW00024023.csv', 'RSM00029467.csv', 'USC00368073.csv', 'USW00024037.csv', 'CHM00054725.csv', 'CHM00050557.csv', 'USW00094910.csv', 'USW00027502.csv', 'USW00093919.csv', 'USC00427271.csv', 'USC00134561.csv', 'USC00132110.csv', 'GME00121078.csv', 'USC00413329.csv', 'USC00166582.csv', 'CHM00058321.csv', 'CHM00051463.csv', 'USW00024221.csv', 'FIE00142526.csv', 'RSM00027648.csv', 'CHM00054527.csv', 'USC00131394.csv', 'RSM00025705.csv', 'USC00318964.csv', 'CHM00053276.csv', 'USC00046136.csv', 'USC00248809.csv', 'USC00222385.csv', 'RSM00029698.csv', 'RSM00031969.csv', 'USC00257040.csv', 'USC00449025.csv', 'RSM00030493.csv', 'USW00014860.csv', 'CA001126150.csv', 'USC00335438.csv', 'USC00450945.csv', 'USW00024021.csv', 'CHM00052418.csv', 'USW00014848.csv', 'CA001021830.csv', 'SW000024180.csv', 'USC00123547.csv', 'CHM00054096.csv', 'GM000010147.csv', 'USC00244558.csv', 'RSM00026781.csv', 'USC00090969.csv', 'USW00013882.csv', 'RSM00034391.csv', 'USW00023160.csv', 'USC00478171.csv', 'USW00014914.csv', 'RSM00029539.csv', 'USC00325993.csv', 'CHM00054135.csv', 'CHM00050353.csv', 'USW00023174.csv', 'RSM00030054.csv', 'LH000026524.csv', 'USW00024155.csv', 'RSM00029263.csv', 'USW00024141.csv', 'CA006085700.csv', 'USC00202445.csv', 'USC00215887.csv', 'RSM00023955.csv', 'USW00014733.csv', 'USC00245040.csv', 'RSM00024790.csv', 'USC00446712.csv', 'USC00381770.csv', 'USC00091732.csv', 'CHM00054337.csv', 'USC00057167.csv', 'USW00025713.csv', 'USC00046730.csv', 'USC00054452.csv', 'USC00424342.csv', 'NOE00109604.csv', 'GME00121126.csv', 'USC00137678.csv', 'USC00350897.csv', 'USC00469281.csv', 'USC00449151.csv', 'ASN00018012.csv', 'USC00311606.csv', 'ASN00094029.csv', 'FR000007190.csv', 'USC00355050.csv', 'USC00332791.csv', 'USC00098703.csv', 'USC00046719.csv', 'ASN00076047.csv', 'RQW00011641.csv', 'USW00014732.csv', 'SPE00120629.csv', 'CA005040680.csv', 'CHM00053698.csv', 'RSM00031439.csv', 'CHM00053673.csv', 'RSM00030727.csv', 'CHM00058040.csv', 'USW00014929.csv', 'CHM00051716.csv', 'FIE00144222.csv', 'USC00272174.csv', 'GME00120946.csv', 'USC00208800.csv', 'CHM00050434.csv', 'USW00023161.csv', 'USW00013883.csv', 'NO000001026.csv', 'USW00013897.csv', 'SWE00138338.csv', 'USW00093729.csv', 'USC00148495.csv', 'CA004056240.csv', 'USC00113320.csv', 'UPM00033587.csv', 'SWE00115916.csv', 'USC00419175.csv', 'USC00356405.csv', 'USC00224939.csv', 'USC00235834.csv', 'CHM00050756.csv', 'CHM00057993.csv', 'USC00087025.csv', 'GM000010393.csv', 'RSM00023405.csv', 'USC00087982.csv', 'CA006078285.csv', 'USW00026410.csv', 'USW00024213.csv', 'USW00023232.csv', 'USC00256385.csv', 'CHM00052602.csv', 'NOE00109768.csv', 'USW00026438.csv', 'RSM00028561.csv', 'USC00147542.csv', 'USC00251415.csv', 'ASN00013017.csv', 'SW000002361.csv', 'USC00394007.csv', 'USC00137700.csv', 'CHM00050968.csv', 'USW00013871.csv', 'USC00114442.csv', 'USW00024011.csv', 'USW00013865.csv', 'RSM00032061.csv', 'USC00050214.csv', 'CA002400600.csv', 'USW00014850.csv', 'GME00102308.csv', 'USW00025333.csv', 'RSM00030879.csv', 'USW00094705.csv', 'JNM00001001.csv', 'USW00093730.csv', 'USC00234919.csv', 'USC00450008.csv', 'USW00093042.csv', 'USC00248597.csv', 'USC00295084.csv', 'USW00014918.csv', 'USC00411875.csv', 'USC00091266.csv', 'USC00091500.csv', 'SWE00137628.csv', 'USC00303025.csv', 'FI000002401.csv', 'USW00013722.csv', 'RSM00024944.csv', 'SPE00119927.csv', 'USW00013736.csv', 'USC00451276.csv', 'RSM00030925.csv', 'USC00456096.csv', 'RSM00022676.csv', 'CHM00051243.csv', 'USC00140365.csv', 'RSM00022892.csv', 'RSM00028367.csv', 'USC00350694.csv', 'USW00003928.csv', 'USC00243617.csv', 'GME00121102.csv', 'CHM00059838.csv', 'RSM00022845.csv', 'USC00470904.csv', 'USC00042598.csv', 'USC00481730.csv', 'RSM00029939.csv', 'RSM00022887.csv', 'USC00028796.csv', 'USC00329430.csv', 'CHM00057633.csv', 'USC00237632.csv', 'GME00114621.csv', 'HRE00105182.csv', 'USC00012813.csv', 'GM000004063.csv', 'RSM00024951.csv', 'USW00013737.csv', 'NLE00102026.csv', 'USC00359316.csv', 'FIE00143596.csv', 'USW00013723.csv', 'USC00317079.csv', 'USC00053867.csv', 'SPE00119729.csv', 'CHM00054662.csv', 'USW00014925.csv', 'RSM00035121.csv', 'USC00041244.csv', 'USW00014919.csv', 'USC00212698.csv', 'USW00023179.csv', 'CA001077500.csv', 'ASN00004032.csv', 'USC00147796.csv', 'USC00034988.csv', 'RSM00023804.csv', 'RSM00026997.csv', 'USC00143527.csv', 'USC00237578.csv', 'USW00014845.csv', 'CHM00050564.csv', 'CHM00053723.csv', 'USW00013870.csv', 'USC00116526.csv', 'CHM00057237.csv', 'USC00221865.csv', 'CHM00052818.csv', 'USC00321871.csv', 'USC00131402.csv', 'USC00250050.csv', 'USC00281351.csv', 'USC00307134.csv', 'NOM00001238.csv', 'USC00473405.csv', 'CA008202000.csv', 'USW00003849.csv', 'USC00165090.csv', 'USW00023233.csv', 'CA006106000.csv', 'USW00026411.csv', 'USC00470124.csv', 'CHM00057745.csv', 'CHM00051334.csv', 'RSM00034163.csv', 'USC00477113.csv', 'USC00403679.csv', 'USC00257715.csv', 'USC00387722.csv', 'RSM00025913.csv', 'SW000002410.csv', 'USW00024012.csv', 'USC00127935.csv', 'USW00013866.csv', 'RSM00022271.csv', 'USC00413873.csv', 'ASN00022008.csv', 'CHM00051644.csv', 'RSM00028748.csv', 'USW00013872.csv', 'NOE00110038.csv', 'ASN00070014.csv', 'USW00014847.csv', 'CHM00056671.csv', 'CA005010640.csv', 'USC00049785.csv', 'USC00322188.csv', 'USW00013899.csv', 'RSM00031594.csv', 'CHM00058847.csv', 'USC00018469.csv', 'CHM00056739.csv', 'USW00023153.csv', 'USC00309292.csv', 'USC00414098.csv', 'USC00016478.csv', 'USC00097600.csv', 'USW00014933.csv', 'RSM00023966.csv', 'USC00390128.csv', 'USC00384607.csv', 'USW00023184.csv', 'RSM00027296.csv', 'USC00235027.csv', 'RSM00034003.csv', 'LOE00105562.csv', 'USC00134381.csv', 'RSM00022113.csv', 'CA001192940.csv', 'RSM00034759.csv', 'RSM00021432.csv', 'USC00030130.csv', 'CHM00057816.csv', 'GME00121114.csv', 'ASN00059040.csv', 'USC00053496.csv', 'USC00043875.csv', 'USC00312102.csv', 'USW00023185.csv', 'USW00013734.csv', 'USW00093134.csv', 'USC00353604.csv', 'ASN00061078.csv', 'USC00047150.csv', 'USC00217907.csv', 'USW00014926.csv', 'USC00342912.csv', 'CHM00058715.csv', 'CHM00053898.csv', 'USW00022516.csv', 'FIE00142106.csv', 'USC00033821.csv', 'USC00319123.csv', 'USW00014891.csv', 'USC00353770.csv', 'IR000040848.csv', 'USW00014852.csv', 'USW00013873.csv', 'USW00013867.csv', 'USC00141858.csv', 'USC00246157.csv', 'RSM00029866.csv', 'USC00243013.csv', 'USC00352693.csv', 'USW00026412.csv', 'SZ000009480.csv', 'CA005052880.csv', 'CHM00058477.csv', 'USW00024229.csv', 'CHM00056462.csv', 'USW00023234.csv', 'CHM00059023.csv', 'RSM00028573.csv', 'RSM00020891.csv', 'USC00114108.csv', 'USC00146128.csv', 'ASN00069018.csv', 'RSM00034172.csv', 'USC00226894.csv', 'USC00470645.csv', 'RSM00029862.csv', 'LU000006590.csv', 'RSM00037123.csv', 'USC00230204.csv', 'USC00238880.csv', 'USW00014842.csv', 'USC00478027.csv', 'FIE00144407.csv', 'USW00025309.csv', 'USC00361705.csv', 'USW00013877.csv', 'NO000099710.csv', 'USW00014895.csv', 'RSM00032098.csv', 'USC00323287.csv', 'USW00014922.csv', 'CA008204700.csv', 'CHM00051709.csv', 'USW00014936.csv', 'RSM00030710.csv', 'GME00102252.csv', 'FIE00142116.csv', 'USW00014739.csv', 'USC00396574.csv', 'CHM00053863.csv', 'SPE00119909.csv', 'AU000015410.csv', 'RSM00029282.csv', 'CA001017230.csv', 'RSM00034984.csv', 'USC00046074.csv', 'USC00394834.csv', 'USC00410984.csv', 'GME00121138.csv', 'USC00398767.csv', 'USC00221094.csv', 'USC00353402.csv', 'USC00251200.csv', 'USC00211630.csv', 'USC00311624.csv', 'USC00203585.csv', 'USW00012836.csv', 'USC00253615.csv', 'NOE00109394.csv', 'CHM00055591.csv', 'CHM00054843.csv', 'USC00332599.csv', 'NOE00133746.csv', 'USW00023194.csv', 'CA002300500.csv', 'USC00112140.csv', 'GME00120970.csv', 'USW00014923.csv', 'USC00428771.csv', 'USW00023157.csv', 'CHM00054102.csv', 'USC00352292.csv', 'USW00094702.csv', 'AU000005010.csv', 'IT000016239.csv', 'USW00013889.csv', 'USW00013876.csv', 'USW00026615.csv', 'CHM00058102.csv', 'USC00049742.csv', 'USW00025308.csv', 'USC00234705.csv', 'CA007113534.csv', 'RSM00023631.csv', 'USW00023023.csv', 'USC00301966.csv', 'SPE00120233.csv', 'CHM00052836.csv', 'ASN00085072.csv', 'CHM00050953.csv', 'FR069029001.csv', 'RSM00020674.csv', 'UK000000000.csv', 'USC00142401.csv', 'CHM00058314.csv', 'USC00132864.csv', 'USW00003859.csv', 'USW00023237.csv', 'USC00236460.csv', 'USC00311975.csv', 'USC00419800.csv', 'FIE00145242.csv', 'RSM00030471.csv', 'FIE00142511.csv', 'USC00297279.csv', 'RSM00034824.csv', 'CA006034075.csv', 'RSM00031960.csv', 'RSM00029849.csv', 'CA008403800.csv', 'USW00014841.csv', 'RSM00027333.csv', 'USW00024028.csv', 'BKM00014654.csv', 'USC00145152.csv', 'USW00026617.csv', 'USW00013874.csv', 'RSM00030673.csv', 'USC00216565.csv', 'USW00093721.csv', 'USW00094728.csv', 'USC00320941.csv', 'USC00444044.csv', 'ASN00003003.csv', 'USC00475474.csv', 'CHM00052203.csv', 'USW00014935.csv', 'ASN00066062.csv', 'USW00023155.csv', 'CA004013660.csv', 'USC00405187.csv', 'USW00094847.csv', 'RSM00027509.csv', 'USC00354811.csv', 'FIE00144202.csv', 'SPE00119711.csv', 'USW00023169.csv', 'USW00013733.csv', 'CHM00058921.csv', 'SWE00139172.csv', 'CHM00051087.csv', 'RSM00027051.csv', 'CHM00059632.csv', 'USC00160098.csv', 'USC00343821.csv', 'USW00012834.csv', 'NOE00109631.csv', 'SWE00115881.csv', 'USC00335747.csv', 'USC00476357.csv', 'RSM00029915.csv', 'USC00247448.csv', 'USC00474582.csv', 'FIE00141810.csv', 'ASN00094008.csv', 'CHM00053487.csv', 'USC00417743.csv', 'USC00147397.csv', 'RSM00031829.csv', 'CA008103200.csv', 'CHM00053336.csv', 'USW00003904.csv', 'RSM00030504.csv', 'RSM00031168.csv', 'RSM00028411.csv', 'CHM00054471.csv', 'SPE00120620.csv', 'USW00023183.csv', 'USC00331905.csv', 'RSM00030935.csv', 'USC00333874.csv', 'USC00126580.csv', 'USC00090140.csv', 'GME00102244.csv', 'USC00473038.csv', 'CHM00053646.csv', 'USW00094846.csv', 'USC00297867.csv', 'USW00023154.csv', 'CHM00054115.csv', 'CHM00059431.csv', 'USW00024149.csv', 'GME00121258.csv', 'ASN00058012.csv', 'USW00093720.csv', 'GME00121270.csv', 'USW00014897.csv', 'IT000016560.csv', 'KG000036974.csv', 'FIE00143381.csv', 'CHM00053915.csv', 'USC00138806.csv', 'USW00026616.csv', 'USC00477464.csv', 'USW00024029.csv', 'USW00023034.csv', 'USW00014840.csv', 'CHM00053068.csv', 'CHM00050788.csv', 'SPE00120224.csv', 'NLE00102427.csv', 'CA007025250.csv', 'GM000003319.csv', 'CHM00050978.csv', 'RSM00023418.csv', 'CHM00054539.csv', 'CHM00054511.csv', 'USC00058429.csv', 'RSM00026275.csv', 'LOE00116344.csv', 'UPM00034523.csv', 'USC00244084.csv', 'LH000026629.csv', 'RSM00027625.csv', 'USC00465563.csv', 'USC00047253.csv', 'USC00048606.csv', 'USC00364432.csv', 'RSM00031735.csv', 'GM000003342.csv', 'SWE00139276.csv', 'CHM00057902.csv', 'USC00367477.csv', 'RSM00026063.csv', 'CHM00051156.csv', 'USW00014827.csv', 'USC00043551.csv', 'CHM00054012.csv', 'USW00014833.csv', 'USW00023047.csv', 'GME00102380.csv', 'USC00295150.csv', 'USW00094014.csv', 'CHM00059287.csv', 'CHM00059293.csv', 'USC00259200.csv', 'CHM00057447.csv', 'USW00013966.csv', 'USC00042713.csv', 'USW00093814.csv', 'USC00364778.csv', 'USC00130385.csv', 'USC00213282.csv', 'USW00013782.csv', 'RSM00031443.csv', 'USC00406371.csv', 'USC00245761.csv', 'USC00042934.csv', 'USW00013741.csv', 'CHM00050658.csv', 'RSM00027037.csv', 'CHM00058238.csv', 'CA003075040.csv', 'USC00092283.csv', 'USC00196783.csv', 'NOE00109680.csv', 'CA003072920.csv', 'USC00306314.csv', 'ASN00089002.csv', 'NLE00101982.csv', 'USC00314063.csv', 'USC00204502.csv', 'USC00254335.csv', 'USC00349364.csv', 'GME00121174.csv', 'USC00332098.csv', 'RSM00031873.csv', 'NO000001465.csv', 'USC00055722.csv', 'USC00316853.csv', 'RSM00029999.csv', 'HR000142360.csv', 'USC00029158.csv', 'USW00014991.csv', 'SPE00119945.csv', 'USW00013740.csv', 'RSM00029594.csv', 'SPE00120134.csv', 'ASN00008051.csv', 'EN000026249.csv', 'NLE00102092.csv', 'RSM00028661.csv', 'RSM00029231.csv', 'USW00093815.csv', 'RSM00029557.csv', 'USC00356907.csv', 'GME00102236.csv', 'CA002202400.csv', 'RSM00027208.csv', 'USW00013967.csv', 'CA004028040.csv', 'NOE00105483.csv', 'USC00147093.csv', 'ROE00108892.csv', 'USC00398307.csv', 'CHM00057297.csv', 'USC00429595.csv', 'USW00014826.csv', 'USC00020949.csv', 'GME00102356.csv', 'FR000007510.csv', 'USC00043578.csv', 'USC00312200.csv', 'NLE00100530.csv', 'SPE00120530.csv', 'SW000021280.csv', 'RSM00023330.csv', 'RSM00034880.csv', 'USC00246862.csv', 'USW00003816.csv', 'USC00231340.csv', 'USW00012919.csv', 'USC00106542.csv', 'RSM00030372.csv', 'CHM00051431.csv', 'RIE00100818.csv', 'USC00118740.csv', 'USC00325479.csv', 'USC00411578.csv', 'CHM00057083.csv', 'CA008501900.csv', 'USC00319357.csv', 'NLE00102457.csv', 'FMW00040504.csv', 'USC00190736.csv', 'GME00102340.csv', 'USW00023044.csv', 'USC00105275.csv', 'USW00023050.csv', 'CHM00051828.csv', 'CHM00053959.csv', 'USC00449263.csv', 'USC00021314.csv', 'USC00106152.csv', 'USC00057309.csv', 'USW00093817.csv', 'RSM00029541.csv', 'USW00094822.csv', 'NLM00006310.csv', 'USC00224377.csv', 'USW00014944.csv', 'RSM00037031.csv', 'USW00013959.csv', 'USW00013781.csv', 'USW00014950.csv', 'CA004047240.csv', 'USC00164700.csv', 'USC00392984.csv', 'USW00014777.csv', 'USC00229860.csv', 'USC00250435.csv', 'IR000040745.csv', 'NOE00112089.csv', 'RSM00034289.csv', 'USW00026510.csv', 'NOE00109640.csv', 'CHM00053352.csv', 'RSM00027008.csv', 'CA006158733.csv', 'USC00329445.csv', 'CA001126510.csv', 'USC00456974.csv', 'USC00297340.csv', 'CHM00053391.csv', 'CHM00050854.csv', 'GME00121162.csv', 'USC00401145.csv', 'USC00033466.csv', 'USC00475808.csv', 'USC00133473.csv', 'USC00029359.csv', 'RSM00032389.csv', 'USC00256290.csv', 'USW00012844.csv', 'CHM00056571.csv', 'RSM00024266.csv', 'FIE00144702.csv', 'USC00048758.csv', 'CA001068130.csv', 'USW00013743.csv', 'RSM00023904.csv', 'CHM00053192.csv', 'CHM00056954.csv', 'USW00013958.csv', 'FIE00145622.csv', 'USC00471676.csv', 'USC00418818.csv', 'USW00094823.csv', 'USC00258915.csv', 'RSM00030777.csv', 'RSM00028138.csv', 'USW00013964.csv', 'USC00411800.csv', 'USC00235253.csv', 'USC00238777.csv', 'SP000003195.csv', 'CHM00056029.csv', 'USW00013970.csv', 'USW00093989.csv', 'USC00160349.csv', 'USC00314464.csv', 'USW00093037.csv', 'NOE00105494.csv', 'GME00102396.csv', 'USC00172765.csv', 'USW00014825.csv', 'CHM00054776.csv', 'RSM00028064.csv', 'CA001016940.csv', 'FMW00040505.csv', 'USC00044957.csv', 'CA006158875.csv', 'RSM00031092.csv', 'SWE00138618.csv', 'USW00093209.csv', 'USC00336389.csv', 'USC00083207.csv', 'USC00366233.csv', 'USW00012924.csv', 'USC00169806.csv', 'SPE00120296.csv', 'USW00012918.csv', 'RSM00031733.csv', 'USC00411596.csv', 'RSM00034139.csv', 'CA006139525.csv', 'USC00133517.csv', 'USC00281335.csv', 'USC00247560.csv', 'USC00331390.csv', 'USC00026132.csv', 'RSM00029418.csv', 'USW00014821.csv', 'USC00111577.csv', 'USC00010655.csv', 'RSM00022217.csv', 'RSM00029430.csv', 'CHM00058606.csv', 'CHM00051811.csv', 'USC00256135.csv', 'NOE00133650.csv', 'USC00047916.csv', 'USW00014606.csv', 'CHM00057290.csv', 'USC00478905.csv', 'USW00014941.csv', 'GME00115771.csv', 'USW00024128.csv', 'USW00093812.csv', 'USW00013960.csv', 'USC00299156.csv', 'USC00232289.csv', 'USC00426919.csv', 'USC00340179.csv', 'USC00160205.csv', 'SWE00139112.csv', 'FIE00143041.csv', 'CHM00052713.csv', 'USW00003959.csv', 'CHM00059134.csv', 'FIE00144712.csv', 'USC00211691.csv', 'GME00121198.csv', 'FIE00142411.csv', 'USC00331541.csv', 'RSM00022820.csv', 'FIE00146688.csv', 'CA006068150.csv', 'USC00452531.csv', 'USC00417079.csv', 'GMW00035032.csv', 'USC00254455.csv', 'GM000004204.csv', 'USC00404987.csv', 'USC00087886.csv', 'UPM00033317.csv', 'CHM00054405.csv', 'USW00026528.csv', 'RSM00028698.csv', 'USC00145906.csv', 'USC00404561.csv', 'CHM00056951.csv', 'RSM00024908.csv', 'USC00393452.csv', 'USC00369298.csv', 'USC00389350.csv', 'USC00473058.csv', 'USC00190120.csv', 'RSM00032165.csv', 'USC00476646.csv', 'RSM00024713.csv', 'USC00315838.csv', 'FIE00145627.csv', 'CHM00056004.csv', 'USC00452914.csv', 'CHM00054161.csv', 'USW00014940.csv', 'USC00431243.csv', 'FRE00104937.csv', 'USC00459074.csv', 'USW00014607.csv', 'USC00045866.csv', 'CHM00058834.csv', 'USW00093026.csv', 'USC00451400.csv', 'RSM00026702.csv', 'GM000004199.csv', 'CA002202200.csv', 'USC00292837.csv', 'USC00054770.csv', 'UK000003026.csv', 'USW00023054.csv', 'USW00014820.csv', 'RSM00025551.csv', 'USC00448192.csv', 'NLM00006260.csv', 'SP000008416.csv', 'GME00102185.csv', 'RSM00031915.csv', 'USW00094239.csv', 'CHM00059082.csv', 'USC00025412.csv', 'USC00327530.csv', 'USC00252805.csv', 'USC00135198.csv', 'RSM00034110.csv', 'SPE00120287.csv', 'USW00012935.csv', 'USC00176937.csv', 'USW00012921.csv', 'ASN00026021.csv', 'CHM00051379.csv', 'RSM00028275.csv', 'RSM00026258.csv', 'ASN00032040.csv', 'USC00085895.csv', 'USC00358494.csv', 'SIM00014015.csv', 'USC00319423.csv', 'CA001021480.csv', 'GME00102193.csv', 'ASN00055024.csv', 'RSM00035026.csv', 'USC00409866.csv', 'USW00014822.csv', 'CHM00056172.csv', 'USC00143554.csv', 'FIE00146258.csv', 'GME00111474.csv', 'USW00014836.csv', 'USW00023042.csv', 'CA008300300.csv', 'RSM00023678.csv', 'ASN00031011.csv', 'FI000000304.csv', 'RSM00030823.csv', 'USC00319147.csv', 'USW00014605.csv', 'USC00397742.csv', 'USW00094830.csv', 'USW00093193.csv', 'CA008104900.csv', 'RSM00030758.csv', 'USW00013963.csv', 'USW00093805.csv', 'USW00013744.csv', 'RSM00027595.csv', 'USC00455704.csv', 'USW00014771.csv', 'USC00305426.csv', 'USC00419532.csv', 'USW00014765.csv', 'USC00368184.csv', 'CA001026270.csv', 'FIE00143056.csv', 'USC00101363.csv', 'USC00098740.csv', 'USC00402489.csv', 'RSM00022837.csv', 'FIE00145382.csv', 'ASN00059030.csv', 'ASN00018044.csv', 'SPE00120332.csv', 'USC00143810.csv', 'USC00369933.csv', 'USC00358536.csv', 'USC00135992.csv', 'USC00137161.csv', 'CHM00057866.csv', 'RSM00026477.csv', 'USC00032794.csv', 'CHM00057127.csv', 'NOE00109653.csv', 'CHM00054374.csv', 'USW00012842.csv', 'USW00014764.csv', 'CHM00054823.csv', 'CHM00057494.csv', 'USC00442245.csv', 'ASN00040004.csv', 'GL000004360.csv', 'IS000009972.csv', 'USW00013976.csv', 'CHM00056985.csv', 'USW00013962.csv', 'NOE00112071.csv', 'USC00036928.csv', 'USW00014943.csv', 'USC00242793.csv', 'RSM00024738.csv', 'USC00055531.csv', 'USC00471923.csv', 'NOE00105492.csv', 'USW00024089.csv', 'USC00200417.csv', 'USC00388887.csv', 'RSM00030822.csv', 'CA006105976.csv', 'USC00244241.csv', 'USC00408562.csv', 'USC00026468.csv', 'CA007066685.csv', 'USW00013802.csv', 'USC00045118.csv', 'USC00051179.csv', 'CA002204100.csv', 'CHM00059254.csv', 'USW00014837.csv', 'USW00094789.csv', 'CA004057120.csv', 'SPE00120521.csv', 'ASN00010579.csv', 'USC00246640.csv', 'USC00418201.csv', 'CA008101500.csv', 'USC00393069.csv', 'USW00003813.csv', 'USW00025624.csv']\n"
     ]
    }
   ],
   "source": [
    "print(\"Bad files:\", len(bad_files))\n",
    "print(bad_files)\n",
    "print(\"Recoverable files:\", len(recoverable_files))\n",
    "print(recoverable_files)\n",
    "print(\"Good files:\", len(good_files))\n",
    "print(good_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
